{
 "cells": [
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "1、引入文档",
   "id": "d1989373420256c9"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "from langchain_community.document_loaders import TextLoader\n",
    "\n",
    "#1、1引入text文档\n",
    "text_documengt=TextLoader(\"knowledge_base/sample.txt\",encoding=\"utf-8\").load()\n",
    "print(text_documengt)"
   ],
   "id": "bb4b9496985d89df",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "\"\"\"\n",
    "UnstructuredMarkdownLoader可用于加载Markdown文件\n",
    "    mode: 加载模式\n",
    "        \"single\"    返回单个Document对象\n",
    "        \"elements\"  按标题等元素切分文档\n",
    "    strategy: 加载策略\n",
    "        \"fast\"      快速粗粒度加载\n",
    "        \"hi_res\"    细粒度加载，按标题层级、列表项、表格等结构细分\n",
    "\"\"\"\n",
    "\n",
    "from langchain_community.document_loaders import UnstructuredMarkdownLoader\n",
    "\n",
    "md_documents = UnstructuredMarkdownLoader(\n",
    "    \"knowledge_base/sample.md\",\n",
    "    mode=\"elements\",\n",
    "    strategy=\"fast\",\n",
    ").load()\n",
    "print(md_documents)"
   ],
   "id": "b1e664a119873b2a",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "\n",
    "#1、3引入pdf文档\n",
    "\n",
    "\"\"\"\n",
    "UnstructuredPDFLoader\n",
    "    支持结构化提取，支持OCR\n",
    "    仅当 PDF 文档中不存在文本时，才会应用 OCR\n",
    "    mode: 加载模式\n",
    "        \"single\"    返回单个Document对象\n",
    "        \"elements\"  按标题等元素切分文档\n",
    "    strategy: 加载策略\n",
    "        \"fast\"      提取并处理文本\n",
    "        \"ocr_only\"  先进行 OCR 处理，再处理原始文本\n",
    "        \"hi_res\"    识别文档布局并处理，自动下载YOLOX模型（识别页面布局）\n",
    "    infer_table_structure: 是否推断表格结构\n",
    "        仅 hi_res 下起效\n",
    "        如果为 True，提取出的表格元素会包含一个 text_as_html 元数据，将文本内容转换为 html 格式\n",
    "    languages: OCR使用的语言\n",
    "        需传入语言列表\n",
    "        语言列表参考 https://github.com/tesseract-ocr/langdata\n",
    "    更多参数详见 https://github.com/Unstructured-IO/unstructured/blob/main/unstructured/partition/pdf.py\n",
    "\"\"\"\n",
    "\n",
    "from langchain_community.document_loaders import UnstructuredPDFLoader\n",
    "\n",
    "pdf_documents = UnstructuredPDFLoader(\n",
    "    \"knowledge_base/sample.pdf\",\n",
    "    mode=\"elements\",\n",
    "    strategy=\"hi_res\",\n",
    "    infer_table_structure=True,\n",
    "    languages=[\"eng\", \"chi_sim\"],\n",
    ").load()\n",
    "print(pdf_documents)"
   ],
   "id": "a953d2fd50ffdefe",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "#1、3引入word文档\n",
    "\n",
    "\"\"\"\n",
    "UnstructuredWordDocumentLoader\n",
    "    适用于 .docx 和 .doc 文件\n",
    "    mode: 加载模式\n",
    "        \"single\"    返回单个Document对象\n",
    "        \"elements\"  按标题等元素切分文档\n",
    "    strategy: 加载策略\n",
    "        \"fast\"      快速粗粒度加载\n",
    "        \"hi_res\"    细粒度加载，按结构细分\n",
    "\"\"\"\n",
    "\n",
    "from langchain_community.document_loaders import UnstructuredWordDocumentLoader\n",
    "\n",
    "word_documents = UnstructuredWordDocumentLoader(\n",
    "    \"knowledge_base/sample.docx\",\n",
    "    mode=\"elements\",\n",
    "    strategy=\"fast\",\n",
    ").load()\n",
    "print(word_documents)"
   ],
   "id": "84193a824821f5c5",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "#1、4通过网页加载文档\n",
    "\"\"\"\n",
    "WebBaseLoader\n",
    "    适用于网页\n",
    "    web_paths: 网址序列\n",
    "    bs_kwargs: 传给 BeautifulSoup 的解析参数\n",
    "        parse_only  只提取指定标签的元素\n",
    "\"\"\"\n",
    "import bs4 #一个用于提取网页数据的库\n",
    "from langchain_community.document_loaders import WebBaseLoader\n",
    "\n",
    "web_documents = WebBaseLoader(\n",
    "    web_paths=(\n",
    "        \" https://news.sina.com.cn/c/xl/2025-09-07/doc-infprmwn0510979.shtml\",\n",
    "    ),\n",
    "    bs_kwargs={\"parse_only\": bs4.SoupStrainer(id=\"article\")},  # 只提取正文区域\n",
    ").load()\n",
    "print(web_documents)"
   ],
   "id": "305b8e76b385e4d2",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-10-31T09:07:18.053248Z",
     "start_time": "2025-10-31T09:01:26.536081Z"
    }
   },
   "cell_type": "code",
   "source": [
    "#1封装加载函数\n",
    "from langchain_community.document_loaders import (\n",
    "    TextLoader,\n",
    "    UnstructuredMarkdownLoader,\n",
    "    UnstructuredPDFLoader,\n",
    "    UnstructuredWordDocumentLoader\n",
    ")\n",
    "\n",
    "def load_documents():\n",
    "    \"\"\"\n",
    "    加载多种类型的文档，包括text、markdown、PDF和Word文档\n",
    "    \n",
    "    Returns:\n",
    "        list: 包含所有加载文档的列表\n",
    "    \"\"\"\n",
    "    # 加载文本文件\n",
    "    text_documents = TextLoader(\n",
    "        \"knowledge_base/sample.txt\",\n",
    "        encoding=\"utf8\"\n",
    "    ).load()\n",
    "\n",
    "    # 加载Markdown文件\n",
    "    md_documents = UnstructuredMarkdownLoader(\n",
    "        \"knowledge_base/sample.md\"\n",
    "    ).load()\n",
    "\n",
    "    # 加载PDF文件\n",
    "    pdf_documents = UnstructuredPDFLoader(\n",
    "        \"knowledge_base/sample.pdf\",\n",
    "        mode=\"elements\",  # 元素模式\n",
    "        strategy=\"hi_res\",  # 高分辨率策略\n",
    "        # strategy=\"fast\",\n",
    "        languages=[\"eng\", \"chi_sim\"],  # 支持的语言：英文和简体中文\n",
    "    ).load()\n",
    "\n",
    "    # 加载Word文档\n",
    "    word_documents = UnstructuredWordDocumentLoader(\n",
    "        \"knowledge_base/sample.docx\"\n",
    "    ).load()\n",
    "\n",
    "    # 返回所有文档的列表\n",
    "    return text_documents + md_documents + pdf_documents + word_documents\n",
    "\n",
    "documents = load_documents()"
   ],
   "id": "a29f5f9a939f87cc",
   "outputs": [],
   "execution_count": 12
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-10-31T11:03:36.123429Z",
     "start_time": "2025-10-31T11:03:36.073563Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import re\n",
    "import json\n",
    "#2、数据处理进行数据清洗\n",
    "def clean_documents(documents: list):\n",
    "    #2、1取出载入的文档，并清洗内容部分\n",
    "    cleaned_documents=[]\n",
    "    for doc in documents:\n",
    "        text=doc.page_content\n",
    "        text=re.sub(r\"\\n{2:}\",\"\\n\\n\",text)\n",
    "        text=text.strip()\n",
    "    #2、2清洗文档的元数据，将元数据装换成chroma支持的类型\n",
    "        allowed_types=(str,int,float,bool)\n",
    "        for key,value in doc.metadata.items():\n",
    "            if not isinstance(value,allowed_types):\n",
    "                try:\n",
    "                    doc.metadata[key]=json.dumps(value,ensure_ascii=False)\n",
    "                except (TypeError,ValueError):\n",
    "                    doc.metadata[key]=str(value)\n",
    "    #2、3将文档重新存入清洗后的序列中\n",
    "        doc.page_content=text\n",
    "        cleaned_documents.append(doc)\n",
    "    return cleaned_documents\n",
    "cleaned_documents=clean_documents(documents)"
   ],
   "id": "72a5d726b0963177",
   "outputs": [],
   "execution_count": 18
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-10-31T11:07:42.102505Z",
     "start_time": "2025-10-31T11:07:42.033850Z"
    }
   },
   "cell_type": "code",
   "source": [
    "#3、递归使用多个分隔符拆分清洗后的文档，拆分成文本块\n",
    "from langchain_text_splitters import RecursiveCharacterTextSplitter\n",
    "\n",
    "# 文本分块\n",
    "text_splitter = RecursiveCharacterTextSplitter(\n",
    "    separators=[\"\\n\\n\", \"。\"],  # 分隔符列表\n",
    "    chunk_size=400,  # 每个块的最大长度\n",
    "    chunk_overlap=40,  # 每个块重叠的长度\n",
    ")\n",
    "texts = text_splitter.split_documents(documents)"
   ],
   "id": "aa7a2c35c72b8ad7",
   "outputs": [],
   "execution_count": 21
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-10-31T11:31:17.823108Z",
     "start_time": "2025-10-31T11:24:56.814102Z"
    }
   },
   "cell_type": "code",
   "source": [
    "#4、载入向量模型并进行向量化\n",
    "import torch\n",
    "from langchain_huggingface import HuggingFaceEmbeddings\n",
    "from langchain_chroma import Chroma\n",
    "#4、1加载向量模型\n",
    "embedding_model=HuggingFaceEmbeddings(\n",
    "    model_name=\"./bge-base-zh-v1.5\",\n",
    "    model_kwargs={\"device\": \"cuda\" if torch.cuda.is_available() else \"cpu\"},\n",
    "    encode_kwargs={\n",
    "        \"normalize_embeddings\": True\n",
    "    },  # 输出归一化向量，更适合余弦相似度计算，控制模长为1\n",
    ")\n",
    "#4.2进行向量化\n",
    "vectorstore=Chroma.from_documents(\n",
    "    texts,\n",
    "    embedding_model,\n",
    "    persist_directory=\"vectorstore\",\n",
    ")"
   ],
   "id": "8ae895c8545af9b4",
   "outputs": [],
   "execution_count": 22
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "#5 、对向量数据库进行添加操作\n",
    "\n",
    "# 5 .1从网页爬取数据并添加到向量数据库中\n",
    "\n",
    "web_document=WebBaseLoader(\n",
    "    web_paths=(\n",
    "        \" https://news.sina.com.cn/c/xl/2025-09-07/doc-infprmwn0510979.shtml\",\n",
    "    ),\n",
    "    bs_kwargs={\"parse_only\": bs4.SoupStrainer(id=\"article\")},  # 只提取正文区域\n",
    ").load()\n",
    "\n",
    "# 5 .2将数据添加到向量数据库中\n",
    "vectorstore.add_documents(web_document)\n",
    "\n",
    "# 5 .3查看向量数据库\n",
    "#5 .3.1按照关键字查询向量文本\n",
    "#print(vectorstore.get(ids='f8492a6e-e8cc-4402-8b84-cddf3dcb7e8f'))\n",
    "\n",
    "#5 .3.2按照关键字查询文档\n",
    "#print(vectorstore.get(where_document={\"$contains\":\"习近平\"}))\n",
    "\n",
    "#5 .4删除向量数据库中的文档\n",
    "vectorstore.delete(ids=['17354737-95cf-49c8-bac9-78452fe8fb4e','454d84b6-2eac-49eb-a034-3181bb1252f3'])\n",
    "print(vectorstore.get(where_document={\"$contains\":\"纪念中国人民\"}))"
   ],
   "id": "89b32cf25ff4bd8d",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-11-03T02:23:28.694319Z",
     "start_time": "2025-11-03T02:23:28.650436Z"
    }
   },
   "cell_type": "code",
   "source": "print(\"hello\")",
   "id": "5cf4623615939fad",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hello\n"
     ]
    }
   ],
   "execution_count": 1
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "rag",
   "language": "python",
   "name": "rag"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.19"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
